package com.chance.cc.crawler.development.scripts.xcar.discounts;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.script.*;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.text.ParseException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class AkScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(AkScript.class);
    private static final String DOMAIN = "xcar";
    private static final String SIDE = "discount";
    private static final String RECORD_AGAIN_REQUEST = "record_again_request";//判断重新下载
    private static final String TURN_PAGE_ITEM_REQUEST = "turn_page_item_request";//判断重新下载
    private static final String PROMOTION = "促销信息";
    private static final String KEYS = "keys";

    private static final String AIKURL = "https://dealer.xcar.com.cn";
    private static final String AKURL = "https://dealer.xcar.com.cn/";

    private static final String AK = "https://dealer.xcar.com.cn/d\\S*/\\S*.htm";
    private static final String SHOPURL = "https://dealer.xcar.com.cn/\\S*/";
    private static final String PROMOTIONURL = "https://dealer.xcar.com.cn/\\S*/news.htm";
    private static final String TURNPAGE = "https://dealer.xcar.com.cn/d\\S*/\\S*.htm\\?type=\\S*";
    private static final String CUXIAO = "https://dealer.xcar.com.cn/\\S*/news_\\S*.htm";
    private static final String DisCount = "https://dealer.xcar.com.cn/\\S*/market\\S*.htm";

    public Map<String, CompiledScript> compiledScriptMap;
    public ScriptEngine scriptEngine;

    public List<String> sizes = new ArrayList<>();
    @Override
    public String domain() {
        return DOMAIN;
    }
    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        String site = crawlerRecord.tagsCreator().bizTags().site();
        return site.equals(SIDE);
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(AKURL);
        addUrlRegular(AK);
        addUrlRegular(SHOPURL);
        addUrlRegular(TURNPAGE);
        addUrlRegular(PROMOTIONURL);
        addUrlRegular(CUXIAO);
        addUrlRegular(DisCount);

    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords){
        String authInfo = getLocalAuthInfo();
        if (StringUtils.isNotBlank(authInfo)){
            crawlerRequestRecord.getHttpRequest().addHeader("cookie",authInfo);
        }
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords){
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")){
                this.initKeyWord(crawlerRequestRecord,supportSourceRecord);
            }
        }
        return prepareLinks;
    }
    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        if (supportSourceRecord == null){return;}
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type+".msg").get();
        if (!"success".equals(msg)){
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> size = new ArrayList<>();
        Map<String,String> map = new HashMap<>();
        List<String> all = json.jsonPath($_type + ".content").all();
        for (String data : all){
            JSONObject jsonObject = JSONObject.parseObject(data);
            JSONObject keyword = jsonObject.getJSONObject("keyword");
            String city = (String) keyword.get("city");
            JSONArray series = keyword.getJSONArray("series");
            for (int i = 0; i <series.size() ; i++) {
                String seriesId = (String) series.get(i);
                String cityAndSeriesId = city+"/"+seriesId;
                size.add(cityAndSeriesId);
                sizes.add(cityAndSeriesId);
            }
        }

        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(KEYS, size);
    }



    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parseLinks = new ArrayList<>();
        String url = crawlerRecord.getHttpRequest().getUrl();
        if ((!page.isDownloadSuccess()) || page.getStatusCode() != 200){
            //log.error("detail  is  [{}]",JSONObject.toJSONString(page));
            log.error("download page !=200 or page ==null"+url+page.getStatusCode());
            if (page.getStatusCode() == 521){
                initCompileScript();//初始化加密方式
                this.cookieUpdate(page,crawlerRecord,parseLinks);
                this.recordAgainRequest(crawlerRecord,parseLinks);
                crawlerRecord.setNeedWashPage(false);
                return parseLinks;
            }else if(page.getStatusCode() != 404){//重新下载
                this.recordAgainRequest(crawlerRecord,parseLinks);
                crawlerRecord.setNeedWashPage(false);
                return parseLinks;
            }
        }
        if (url.matches(AKURL)){
            KVTag kvTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag(KEYS);
            if (kvTag == null){
                crawlerRecord.tagsCreator().bizTags().addCustomKV(KEYS,sizes);
            }
            this.CityAndSeries(crawlerRecord,page);//生成车系和城市的字典表
            this.getSeriesUrl(crawlerRecord,page,parseLinks);//得到具体url
        }
        if (url.matches(AK) || url.matches(TURNPAGE)){//解析页面  返回商铺信息以及url
            this.parseLinksShop(crawlerRecord,page,parseLinks);
        }
        if(url.matches(SHOPURL) && (!url.matches(PROMOTIONURL))){//进入店铺页面，解析促销链接
            this.parseLinksPromotion(crawlerRecord,page,parseLinks);
        }
        if (url.matches(PROMOTIONURL)){//进入促销页面
            this.moduleAll(crawlerRecord,page,parseLinks);
        }
        if (url.matches(CUXIAO)){//获取正在促销的链接
            this.parseLinksPromotionMessage(crawlerRecord,page,parseLinks);
        }
        return parseLinks;
    }

    private void parseLinksPromotionMessage(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        List<Selectable> nodes = page.getHtml().xpath("/html/body/div[6]/div[1]/div[3]/div").nodes();
        //获取优惠倒计时页面
        for (Selectable node : nodes){
            String text = node.xpath("./div[2]/div/i//text()").get();//标识是否是倒计时优惠活动
            if (text != null){
                String articleUrl = node.xpath("./div[2]/span/a/@href").get();
                articleUrl = AIKURL + articleUrl;
                CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRecord)
                        .httpUrl(articleUrl)
                        .recordKey(articleUrl)
                        .releaseTime(System.currentTimeMillis())
                        .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                        .copyBizTags()
                        .copyResultTags()
                        .build();
                parseLinks.add(record);
            }
        }
        //获取下一页
        List<Selectable> nodes1 = page.getHtml().xpath("/html/body/div[6]/div[1]/div[4]/a").nodes();
        String s = "/\\S*/news\\S*.htm";
        for (int i = 0; i <nodes1.size() ; i++) {
            if (i == nodes1.size()-1){
                String turnPage = nodes1.get(i).xpath("./@href").get();
                if (turnPage.matches(s)){
                    turnPage = AIKURL + turnPage;
                    CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRecord)
                            .httpUrl(turnPage)
                            .recordKey(turnPage)
                            .releaseTime(System.currentTimeMillis())
                            .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                            .copyBizTags()
                            .copyResultTags()
                            .build();
                    parseLinks.add(record);
                }
            }
        }

    }

    /*
    * 解析优惠促销模块的url
    * */
    private void moduleAll(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        String url = crawlerRecord.getHttpRequest().getUrl();//供应商url
        List<Selectable> nodes = page.getHtml().xpath("/html/body/div[6]/div[1]/div[2]/a").nodes();
        for (Selectable node : nodes){
            String message = node.xpath("./text()").get();
            if (!"优惠促销".equals(message)){
                continue;
            }
            String  moduleUrl = node.xpath("./@href").get();
            moduleUrl = AIKURL + moduleUrl;
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(moduleUrl)
                    .recordKey(moduleUrl)
                    .releaseTime(System.currentTimeMillis())
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parseLinks.add(record);
        }
    }

    /*
    进入店铺页面，解析促销链接
    * */
    private void parseLinksPromotion(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        List<Selectable> nodes = page.getHtml().xpath("/html/body/div[4]/div/ul/li").nodes();
        for (Selectable node : nodes){
            String promotion = node.xpath("./a//text()").get();
            if (promotion.equals(PROMOTION)){//判断是否具有促销信息模块
                String  promotionUrl = node.xpath("./a/@href").get();
                promotionUrl = AIKURL +promotionUrl;
                CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRecord)
                        .httpUrl(promotionUrl)
                        .recordKey(promotionUrl)
                        .releaseTime(System.currentTimeMillis())
                        .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                        .copyBizTags()
                        .copyResultTags()
                        .build();
                parseLinks.add(record);
            }else{
                continue;
            }
        }
    }

    /*
    * 解析页面  返回商铺信息以及url
    * */
    private void parseLinksShop(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        List<String> kindAll = page.getHtml().xpath("//*[@id=\"dealer_list_tab\"]/li").all();
        List<Map<String,String>> shopAllMessageMap = new ArrayList<>();
        if (kindAll.size() == 1){//代表只有4s店
            this.parseLinksShop4s(crawlerRecord,page,parseLinks);
        }else if (!crawlerRecord.getHttpRequest().getUrl().matches(TURNPAGE)){//代表既有4s店又有综合店
            this.parseLinksShop4s(crawlerRecord,page,parseLinks);
            this.parseLinksShopAll(crawlerRecord,page,parseLinks);
        }else{
            this.parseLinksShop4s(crawlerRecord,page,parseLinks);
        }
        crawlerRecord.tagsCreator().bizTags().addCustomKV("shopAllMessageMap",shopAllMessageMap);//得到所有店铺的url
    }
    //解析综合店模块
    private void parseLinksShopAll(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        List<Selectable> shopAll = page.getHtml().xpath("//*[@id=\"dlists_zh\"]/ul/li").nodes();
        this.shopAllContent(shopAll,crawlerRecord,page,parseLinks);
        /*String text = page.getHtml().xpath("//*[@id=\"dlists_zh\"]/div/a//text()").get();//判断是否存在下一页
        if (text != null){}*/
    }
    //解析4s店模块
    private void parseLinksShop4s(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        List<Selectable> shopAll = page.getHtml().xpath("//*[@id=\"dlists_4s_isfee\"]/ul/li").nodes();
        this.shopAllContent(shopAll,crawlerRecord,page,parseLinks);
        String text = page.getHtml().xpath("//*[@id=\"dlists_4s_isfee\"]/div/a//text()").get();//判断是否存在下一页
        if (text != null){
            List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"dlists_4s_isfee\"]/div/a").nodes();//解析下一页
            String span = null;
            String span1 = page.getHtml().xpath("//*[@id=\"dlists_4s_isfee\"]/div/span[1]//text()").get();
            if (span1.equals("...")){
                span =null;
            }else{
                span = span1;
            }
            String span2 = page.getHtml().xpath("//*[@id=\"dlists_4s_isfee\"]/div/span[2]//text()").get();//获得当前第几页
            if (span2.equals("...")){
                span =null;
            }else{
                span = span2;
            }
            if(span.equals(String.valueOf(nodes.size()))){
                return;
            }
            Selectable node = nodes.get(nodes.size()-1);//获取下一页
            String turnPageUrl = node.xpath("./@href").get();
            String url = crawlerRecord.getHttpRequest().getUrl();
            String start = turnPageUrl.substring(7);
            String end = start.substring(0,start.length()-22);
            if (end.length() == 1){
                String s1 = turnPageUrl.substring(0,20);
                String s2 = turnPageUrl.substring(24,turnPageUrl.length());
                turnPageUrl = AIKURL + (s1+s2);
            }else if (end.length() == 2){
                String s1 = turnPageUrl.substring(0, 18);
                String s2 = turnPageUrl.substring(25,turnPageUrl.length());
                turnPageUrl = AIKURL + (s1+"=1&"+s2);
            }else if (end.length() == 3){
                String s1 = turnPageUrl.substring(0,22);
                String s2 = turnPageUrl.substring(26,turnPageUrl.length());
                turnPageUrl = AIKURL + (s1+s2);
            }else if (end.length() == 4){
                String s1 = turnPageUrl.substring(0,23);
                String s2 = turnPageUrl.substring(27,turnPageUrl.length());
                turnPageUrl = AIKURL + (s1+s2);
            }

            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(turnPageUrl)
                    .recordKey(turnPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parseLinks.add(record);
        }

    }
    //解析出商铺的所有内容
    private void shopAllContent(List<Selectable> shopAll, CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        for (Selectable shop:shopAll){
            String shopUrl = shop.xpath("./dl/dt/a/@href").get();//商铺url
            String shopId = shopUrl.substring(1,shopUrl.length()-1);//商铺id
            shopUrl = AIKURL + shopUrl;
            String shopName = shop.xpath("./dl/dt/a//text()").get();//商铺名称
            String shopPhone = shop.xpath("./dl/dd[1]/em/b//text()").get(); //商铺电话
            if (StringUtils.isBlank(shopPhone)){
                shopPhone = shop.xpath("./dl/dd[1]/em/b/i[2]//text()").get(); //商铺电话
            }
            String shopAddress = shop.xpath("./dl/dd[2]/span[2]//text()").get();//商铺地址
            Map<String,String> Tag_Shop_detail = new HashMap<>();//存放商铺信息
            Tag_Shop_detail.put("Field_Shop_id",shopId);
            Tag_Shop_detail.put("Field_Shop_Url",shopUrl);
            Tag_Shop_detail.put("Field_Shop_Name",shopName);
            Tag_Shop_detail.put("Field_Shop_Phone",shopPhone);
            Tag_Shop_detail.put("Field_Shop_Address",shopAddress);
            crawlerRecord.tagsCreator().bizTags().addCustomKV("Tag_Shop_detail",Tag_Shop_detail);
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(shopUrl)
                    .recordKey(shopUrl)
                    .releaseTime(System.currentTimeMillis())
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parseLinks.add(record);
        }
    }
    /*
    * 得到具体url
    * */
    private void getSeriesUrl(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        KVTag k1 = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("cityMap");
        KVTag k2 = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("seriesMap");
        Map<String,String> cityMap = (Map<String, String>) k1.getVal();
        Map<String,String> seriesMap = (Map<String, String>) k2.getVal();
        KVTag kvTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag(KEYS);//得到城市
        List<String> size = (List<String>) kvTag.getVal();
        if (size.size() != 0){
            for (int i = 0; i <size.size() ; i++) {
                String cityAndSeries = size.get(i);
                log.info(cityAndSeries);
                String[] split1 = cityAndSeries.split("/");
                String city = split1[0];//得到城市
                String series = split1[1];//得到车系
                String cityNumber= cityMap.get(city);//得到城市代表的数字
                String seriesUrl= seriesMap.get(series);//得到车系代表的半个url
                String[] split = seriesUrl.split("/");
                String url = AKURL+"d"+cityNumber+"/"+split[2];
                String seriesId = split[2].substring(0,(split[2].length())-4);//车系id
                Map<String,String> Tag_Brand_detail = new HashMap<>();//品牌详情
                Tag_Brand_detail.put("series",series);
                Tag_Brand_detail.put("seriesId",seriesId);
                crawlerRecord.tagsCreator().bizTags().getCategoryTag().addKVTag("cityMap",null);
                crawlerRecord.tagsCreator().bizTags().getCategoryTag().addKVTag("seriesMap",null);
                CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRecord)
                        .httpUrl(url)
                        .recordKey(url)
                        .releaseTime(System.currentTimeMillis())
                        .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                        .copyResultTags()
                        .copyBizTags()
                        .build();
                record.tagsCreator().bizTags().addCustomKV("Brand_series_url",url);
                record.tagsCreator().bizTags().addCustomKV("Tag_City",city);//城市名称
                record.tagsCreator().bizTags().addCustomKV("Tag_Brand_detail",Tag_Brand_detail);//品牌详情
                parseLinks.add(record);
            }
        }else{
            String firstCity = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("firstCity");
            String firstBrand = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("firstBrand");
            if (firstCity ==null && firstBrand == null){
                return;
            }
            String cityNumber= cityMap.get(firstCity);//得到城市代表的数字
            String seriesUrl= seriesMap.get(firstBrand);//得到车系代表的半个url
            String[] split = seriesUrl.split("/");
            String url = AKURL+"d"+cityNumber+"/"+split[2];
            String seriesId = split[2].substring(0,(split[2].length())-4);//车系id
            Map<String,String> Tag_Brand_detail = new HashMap<>();//品牌详情
            Tag_Brand_detail.put("series",firstBrand);
            Tag_Brand_detail.put("seriesId",seriesId);
            crawlerRecord.tagsCreator().bizTags().getCategoryTag().addKVTag("cityMap",null);
            crawlerRecord.tagsCreator().bizTags().getCategoryTag().addKVTag("seriesMap",null);
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(url)
                    .recordKey(url)
                    .releaseTime(System.currentTimeMillis())
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .copyResultTags()
                    .copyBizTags()
                    .build();
            record.tagsCreator().bizTags().addCustomKV("Brand_series_url",url);
            record.tagsCreator().bizTags().addCustomKV("Tag_City",firstCity);//城市名称
            record.tagsCreator().bizTags().addCustomKV("Tag_Brand_detail",Tag_Brand_detail);//品牌详情
            parseLinks.add(record);

        }
    }

    //生成车系和城市的字典表
    private void CityAndSeries(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        Map<String,String> cityMap = new HashMap<>();
        Map<String,String> seriesMap = new HashMap<>();
        List<Selectable> cityAll = page.getHtml().xpath("//*[@id=\"select_province\"]/li").nodes();
        for (Selectable city : cityAll ){
            String cityNumber = city.xpath("./@value").get();
            String cityName = city.xpath("./a//text()").get();
            cityMap.put(cityName,cityNumber);
        }
        List<Selectable> seriesAll = page.getHtml().xpath("/html/body/div[7]/div/div[2]/dl/dd/div").nodes();
        int count = 0;
        for (Selectable series : seriesAll){
            if (count == 0){count++; continue;}else{
                List<Selectable> nodes = series.xpath("./a").nodes();
                count++;
                for (Selectable node : nodes){
                    String seriesUrl = node.xpath("./@href").get();
                    String seriesName = node.xpath(".//text()").get();
                    seriesMap.put(seriesName,seriesUrl);
                }
            }
        }
        crawlerRecord.tagsCreator().bizTags().addCustomKV("cityMap",cityMap);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("seriesMap",seriesMap);
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        String url = crawlerRecord.getHttpRequest().getUrl();
        List<CrawlerData> crawlerData = new ArrayList<>();
        if (crawlerRecord.tagsCreator().resultTags().hasDataType(CrawlerEnum.CrawlerDataType.article)){
            if (url.matches(DisCount)){
                this.washDisCount(crawlerRecord,page,crawlerData);
            }
        }
        return crawlerData;
    }
    //清洗信息
    private void washDisCount(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerData> crawlerData){
        String url = crawlerRecord.getHttpRequest().getUrl();//文章链接
        String dealer = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("ak47");//经销商
        String city = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("Tag_City");//城市名称
        String brand_series_url = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("Brand_series_url");//品牌url
        KVTag tag_brand_detail = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("Tag_Brand_detail");//车系
        Map<String,String> val1 = (Map<String, String>) tag_brand_detail.getVal();
        String brand = val1.get("series");//车系品牌名称
        String seriesId = val1.get("seriesId");//品牌id
        KVTag kvTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("Tag_Shop_detail");//商铺的信息
        Map<String,String> val = (Map<String, String>) kvTag.getVal();
        String field_shop_id = val.get("Field_Shop_id");//商铺id
        String field_shop_url = val.get("Field_Shop_Url");//商铺链接
        String field_shop_name = val.get("Field_Shop_Name");//商铺名称
        //String field_shop_phone = val.get("Field_Shop_Phone");//商铺电话
        String field_shop_phone = page.getHtml().xpath("/html/body/div[3]/div/dl/dd/div/b//text()").get();//商铺电话
        String field_shop_address = val.get("Field_Shop_Address");//商铺地址
        String categroy_1st = page.getHtml().xpath("/html/body/div[5]/a[2]//text()").get();//促销信息
        String categroy_2st = page.getHtml().xpath("/html/body/div[5]/a[3]//text()").get();//优惠促销
        List<String> all = page.getHtml().xpath("/html/body/div[6]/div[1]//img/@src").all();//获取到页面所有图片
        StringBuilder images = new StringBuilder();
        for (String img : all){
            images.append(img).append("\\x01");
        }
        String title = page.getHtml().xpath("/html/body/div[6]/div[1]/span//text()").get();//标题
        String date = page.getHtml().xpath("/html/body/div[6]/div[1]/div[1]/em//text()").get();//活动日期
        String[] split = date.split("：");
        String releaseTime1 = split[1].substring(0,10);
        Long release = 0l;
        try {
            release = DateUtils.parseDate(releaseTime1, "yyyy-MM-dd").getTime();
        } catch (ParseException e) {
            log.error("article time Parsing time error");
        }

        String date1 = date.substring(5,date.length());
        String time = page.getHtml().xpath("/html/body/div[6]/div[1]/div[1]/i//text()").get();//倒计时
        String content = page.getHtml().xpath("/html/body/div[6]/div[1]/p[1]").get();//内容
        crawlerRecord.tagsCreator().bizTags().addCustomKV("Tag_Shop_detail",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("Tag_Brand_detail",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("Brand_series_url",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("Tag_City",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("ak47",null);
        CrawlerData crawlerData1 = CrawlerData.builder()
                .data(crawlerRecord,page)
                .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article,new Random().nextInt(1000000000)))
                .releaseTime(release)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .url(url)
                .addContentKV("platform",dealer)
                .addContentKV(AICCommonField.Field_Images, String.valueOf(images))
                .addContentKV(AutoVMCommonField.Tag_City,city)
                .addContentKV(AutoVMCommonField.Field_Brand_name,brand)
                .addContentKV(AutoVMCommonField.Field_Brand_url,brand_series_url)
                .addContentKV("categroy_1st",categroy_1st)
                .addContentKV("categroy_2st",categroy_2st)
                .addContentKV(AutoVMCommonField.Field_Shop_Url,field_shop_url)
                .addContentKV(AutoVMCommonField.Field_Shop_Name,field_shop_name)
                .addContentKV(AutoVMCommonField.Field_Shop_Phone,field_shop_phone)
                .addContentKV(AutoVMCommonField.Field_Shop_Address,field_shop_address)
                .addContentKV(AutoVMCommonField.Field_Brand_id,seriesId)
                .addContentKV("promotion_period",date1)
                .addContentKV("count_down",time)
                .addContentKV(AICCommonField.Field_Title,title)
                .addContentKV(AICCommonField.Field_Content,content)
                .flowInPipelineTag("kafka")
                .build();
        crawlerData1.setFilterPipelineResult(true);
        crawlerData.add(crawlerData1);
    }


    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }
    /*
     * 下在失败 重新下载
     * */
    private void recordAgainRequest(CrawlerRequestRecord crawlerRecord, List<CrawlerRequestRecord> linksRecords) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(RECORD_AGAIN_REQUEST)){//判断是否重新下载过
            count = (int)crawlerBusinessTags.getCategoryTag().getKVTag(RECORD_AGAIN_REQUEST).getVal();
            if (count >= 5){
                log.error(DOMAIN + " download page the number of retries exceeds the limit,request url {}",crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }
        String url = crawlerRecord.getHttpRequest().getUrl();
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get(TURN_PAGE_ITEM_REQUEST);
        CrawlerRequestRecord crawlerRequestRecord = null;
        if (type == null){
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .releaseTime(System.currentTimeMillis())
                    .httpUrl(url)
                    .recordKey(crawlerRecord.getRecordKey()+count)
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }else{
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .releaseTime(System.currentTimeMillis())
                    .httpUrl(url)
                    .recordKey(crawlerRecord.getRecordKey()+count)
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .copyBizTags()
                    .copyRequestTags()
                    .build();
        }
        if (crawlerRequestRecord == null){
            return;
        }
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(RECORD_AGAIN_REQUEST,count);
        linksRecords.add(crawlerRequestRecord);
    }

    private void cookieUpdate(HttpPage page, CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> parseLinks){
        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());
        HttpPage httpPage = downloader.download(crawlerRequestRecord.getHttpRequest(), crawlerRequestRecord.getHttpConfig());
        String js = httpPage.getRawText().substring(httpPage.getRawText().indexOf("("), httpPage.getRawText().lastIndexOf(")")+1);
        Map<String,String> cookieMap = new HashMap<>();
        try {
            String eval = (String) scriptEngine.eval(js);
            eval =  eval.substring(eval.indexOf("=")+1, eval.indexOf(";"));
            cookieMap.put("__jsl_clearance_s",eval);
            log.info("xcar cookie second update result __jsl_clearance_s={}",eval);
            crawlerRequestRecord.getHttpRequest().addHeader("cookie",processCookie(cookieMap));

            // 2. 生成 第二个 加密 cookie
            httpPage = downloader.download(crawlerRequestRecord.getHttpRequest(), crawlerRequestRecord.getHttpConfig());
            Matcher mtAuthor = Pattern.compile("};go\\((.*?)\\)</script>").matcher(httpPage.getRawText());
            if (mtAuthor.find()){
                String cookieGen = mtAuthor.group(1);
                Json cookieGenJson = new Json(cookieGen);
                String ha = cookieGenJson.jsonPath($_type + ".ha").get();
                log.info("xcar cookie cipher code {}",ha);
                compiledScriptMap.get(ha).eval();
                Invocable invocable = (Invocable) compiledScriptMap.get(ha).getEngine();
                String result = (String) invocable.invokeFunction("go", JSON.parseObject(cookieGen, Map.class));
                result =  result.substring(result.indexOf("=")+1, result.indexOf(";"));
                cookieMap.put("__jsl_clearance_s",result);
                result = processCookie(cookieMap);
                updateLocalAuthInfo(result);
                log.info("xcar cookie second update result {}",result);
            }

        } catch (Exception e) {
            log.error("xcar cookie update error {}",e.getMessage());
        }

    }
    private String processCookie(Map<String,String> cookieMap){
        StringBuffer cookieSB = new StringBuffer();
        for (Map.Entry<String, String> entry : cookieMap.entrySet()) {
            cookieSB.append(entry.getKey())
                    .append("=")
                    .append(entry.getValue())
                    .append(";");
        }
        return cookieSB.substring(0,cookieSB.lastIndexOf(";"));
    }


    private void initCompileScript(){
        if (compiledScriptMap == null){
            compiledScriptMap = new HashMap<>();
        }

        ScriptEngineManager sm = new ScriptEngineManager();
        NashornScriptEngineFactory factory = null;
        for (ScriptEngineFactory f : sm.getEngineFactories()) {
            if (f.getEngineName().equalsIgnoreCase("Oracle Nashorn")) {
                factory = (NashornScriptEngineFactory)f;
                break;
            }
        }
        String[] stringArray = ArrayUtils.toArray("-doe", "--global-per-engine");
        scriptEngine = factory.getScriptEngine(stringArray);
        List<String> ciphers = Arrays.asList("md5", "sha1", "sha256");
        String filePathPrefix = "E:\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\xcar\\js\\";
      // String filePathPrefix = "/data/chance_crawler_runner/domain/xcar/cipher_js/";
        for (String cipher : ciphers) {
            String filePath = filePathPrefix+cipher+".js";
            log.info("xcar cipher js file path {}",filePath);
            final CompiledScript compiled;
            try {
                compiled = ((Compilable)scriptEngine).compile(new FileReader(filePath));
                compiledScriptMap.put(cipher,compiled);
            } catch (ScriptException e) {
                e.printStackTrace();
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }

        }
    }


}
